{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.distributions import Normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Use CUDA</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "device   = torch.device(\"cuda\" if use_cuda else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Replay Buffer</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer:\n",
    "    def __init__(self, capacity):\n",
    "        self.capacity = capacity\n",
    "        self.buffer = []\n",
    "        self.position = 0\n",
    "    \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        if len(self.buffer) < self.capacity:\n",
    "            self.buffer.append(None)\n",
    "        self.buffer[self.position] = (state, action, reward, next_state, done)\n",
    "        self.position = (self.position + 1) % self.capacity\n",
    "    \n",
    "    def sample(self, batch_size):\n",
    "        batch = random.sample(self.buffer, batch_size)\n",
    "        state, action, reward, next_state, done = map(np.stack, zip(*batch))\n",
    "        return state, action, reward, next_state, done\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.buffer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Normalize action space</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NormalizedActions(gym.ActionWrapper):\n",
    "\n",
    "    def _action(self, action):\n",
    "        low_bound   = self.action_space.low\n",
    "        upper_bound = self.action_space.high\n",
    "        \n",
    "        action = low_bound + (action + 1.0) * 0.5 * (upper_bound - low_bound)\n",
    "        action = np.clip(action, low_bound, upper_bound)\n",
    "        \n",
    "        return action\n",
    "\n",
    "    def _reverse_action(self, action):\n",
    "        low_bound   = self.action_space.low\n",
    "        upper_bound = self.action_space.high\n",
    "        \n",
    "        action = 2 * (action - low_bound) / (upper_bound - low_bound) - 1\n",
    "        action = np.clip(action, low_bound, upper_bound)\n",
    "        \n",
    "        return actions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Ornstein-Uhlenbeck process</h2>\n",
    "Adding time-correlated noise to the actions taken by the deterministic policy<br>\n",
    "<a href=\"https://en.wikipedia.org/wiki/Ornstein%E2%80%93Uhlenbeck_process\">wiki</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OUNoise(object):\n",
    "    def __init__(self, action_space, mu=0.0, theta=0.15, max_sigma=0.3, min_sigma=0.3, decay_period=100000):\n",
    "        self.mu           = mu\n",
    "        self.theta        = theta\n",
    "        self.sigma        = max_sigma\n",
    "        self.max_sigma    = max_sigma\n",
    "        self.min_sigma    = min_sigma\n",
    "        self.decay_period = decay_period\n",
    "        self.action_dim   = action_space.shape[0]\n",
    "        self.low          = action_space.low\n",
    "        self.high         = action_space.high\n",
    "        self.reset()\n",
    "        \n",
    "    def reset(self):\n",
    "        self.state = np.ones(self.action_dim) * self.mu\n",
    "        \n",
    "    def evolve_state(self):\n",
    "        x  = self.state\n",
    "        dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.action_dim)\n",
    "        self.state = x + dx\n",
    "        return self.state\n",
    "    \n",
    "    def get_action(self, action, t=0):\n",
    "        ou_state = self.evolve_state()\n",
    "        self.sigma = self.max_sigma - (self.max_sigma - self.min_sigma) * min(1.0, t / self.decay_period)\n",
    "        return np.clip(action + ou_state, self.low, self.high)\n",
    "    \n",
    "#https://github.com/vitchyr/rlkit/blob/master/rlkit/exploration_strategies/ou_strategy.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(frame_idx, rewards):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, rewards[-1]))\n",
    "    plt.plot(rewards)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> Continuous control with deep reinforcement learning</h1>\n",
    "<h2><a href=\"https://arxiv.org/abs/1509.02971\">Arxiv</a></h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ValueNetwork(nn.Module):\n",
    "    def __init__(self, num_inputs, num_actions, hidden_size, init_w=3e-3):\n",
    "        super(ValueNetwork, self).__init__()\n",
    "        \n",
    "        self.linear1 = nn.Linear(num_inputs + num_actions, hidden_size)\n",
    "        self.linear2 = nn.Linear(hidden_size, hidden_size)\n",
    "        self.linear3 = nn.Linear(hidden_size, 1)\n",
    "        \n",
    "        self.linear3.weight.data.uniform_(-init_w, init_w)\n",
    "        self.linear3.bias.data.uniform_(-init_w, init_w)\n",
    "        \n",
    "    def forward(self, state, action):\n",
    "        x = torch.cat([state, action], 1)\n",
    "        x = F.relu(self.linear1(x))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        x = self.linear3(x)\n",
    "        return x\n",
    "    \n",
    "\n",
    "class PolicyNetwork(nn.Module):\n",
    "    def __init__(self, num_inputs, num_actions, hidden_size, init_w=3e-3):\n",
    "        super(PolicyNetwork, self).__init__()\n",
    "        \n",
    "        self.linear1 = nn.Linear(num_inputs, hidden_size)\n",
    "        self.linear2 = nn.Linear(hidden_size, hidden_size)\n",
    "        self.linear3 = nn.Linear(hidden_size, num_actions)\n",
    "        \n",
    "        self.linear3.weight.data.uniform_(-init_w, init_w)\n",
    "        self.linear3.bias.data.uniform_(-init_w, init_w)\n",
    "        \n",
    "    def forward(self, state):\n",
    "        x = F.relu(self.linear1(state))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        x = F.tanh(self.linear3(x))\n",
    "        return x\n",
    "    \n",
    "    def get_action(self, state):\n",
    "        state  = torch.FloatTensor(state).unsqueeze(0).to(device)\n",
    "        action = self.forward(state)\n",
    "        return action.detach().cpu().numpy()[0, 0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>DDPG Update</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ddpg_update(batch_size, \n",
    "           gamma = 0.99,\n",
    "           min_value=-np.inf,\n",
    "           max_value=np.inf,\n",
    "           soft_tau=1e-2):\n",
    "    \n",
    "    state, action, reward, next_state, done = replay_buffer.sample(batch_size)\n",
    "    \n",
    "    state      = torch.FloatTensor(state).to(device)\n",
    "    next_state = torch.FloatTensor(next_state).to(device)\n",
    "    action     = torch.FloatTensor(action).to(device)\n",
    "    reward     = torch.FloatTensor(reward).unsqueeze(1).to(device)\n",
    "    done       = torch.FloatTensor(np.float32(done)).unsqueeze(1).to(device)\n",
    "\n",
    "    policy_loss = value_net(state, policy_net(state))\n",
    "    policy_loss = -policy_loss.mean()\n",
    "\n",
    "    next_action    = target_policy_net(next_state)\n",
    "    target_value   = target_value_net(next_state, next_action.detach())\n",
    "    expected_value = reward + (1.0 - done) * gamma * target_value\n",
    "    expected_value = torch.clamp(expected_value, min_value, max_value)\n",
    "\n",
    "    value = value_net(state, action)\n",
    "    value_loss = value_criterion(value, expected_value.detach())\n",
    "\n",
    "\n",
    "    policy_optimizer.zero_grad()\n",
    "    policy_loss.backward()\n",
    "    policy_optimizer.step()\n",
    "\n",
    "    value_optimizer.zero_grad()\n",
    "    value_loss.backward()\n",
    "    value_optimizer.step()\n",
    "\n",
    "    for target_param, param in zip(target_value_net.parameters(), value_net.parameters()):\n",
    "            target_param.data.copy_(\n",
    "                target_param.data * (1.0 - soft_tau) + param.data * soft_tau\n",
    "            )\n",
    "\n",
    "    for target_param, param in zip(target_policy_net.parameters(), policy_net.parameters()):\n",
    "            target_param.data.copy_(\n",
    "                target_param.data * (1.0 - soft_tau) + param.data * soft_tau\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "env = NormalizedActions(gym.make(\"Pendulum-v0\"))\n",
    "ou_noise = OUNoise(env.action_space)\n",
    "\n",
    "state_dim  = env.observation_space.shape[0]\n",
    "action_dim = env.action_space.shape[0]\n",
    "hidden_dim = 256\n",
    "\n",
    "value_net  = ValueNetwork(state_dim, action_dim, hidden_dim).to(device)\n",
    "policy_net = PolicyNetwork(state_dim, action_dim, hidden_dim).to(device)\n",
    "\n",
    "target_value_net  = ValueNetwork(state_dim, action_dim, hidden_dim).to(device)\n",
    "target_policy_net = PolicyNetwork(state_dim, action_dim, hidden_dim).to(device)\n",
    "\n",
    "for target_param, param in zip(target_value_net.parameters(), value_net.parameters()):\n",
    "    target_param.data.copy_(param.data)\n",
    "\n",
    "for target_param, param in zip(target_policy_net.parameters(), policy_net.parameters()):\n",
    "    target_param.data.copy_(param.data)\n",
    "    \n",
    "    \n",
    "value_lr  = 1e-3\n",
    "policy_lr = 1e-4\n",
    "\n",
    "value_optimizer  = optim.Adam(value_net.parameters(),  lr=value_lr)\n",
    "policy_optimizer = optim.Adam(policy_net.parameters(), lr=policy_lr)\n",
    "\n",
    "value_criterion = nn.MSELoss()\n",
    "\n",
    "replay_buffer_size = 1000000\n",
    "replay_buffer = ReplayBuffer(replay_buffer_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_frames  = 12000\n",
    "max_steps   = 500\n",
    "frame_idx   = 0\n",
    "rewards     = []\n",
    "batch_size  = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAE/CAYAAABLrsQiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsvXl8Y2d56P99JFmyJK+yZ/UynkwmewhJJjNkhxBCuC0NBdoCBQJdWEp7of2Ve+ntTqELvbe9UAptKEspULbLEtpASggQsmcSSDKTbfYZj2fsseVVkrW+vz/OOfKxrM22ZFvj5/v56GPpPYveI0vvc55djDEoiqIo6xfPak9AURRFWV1UECiKoqxzVBAoiqKsc1QQKIqirHNUECiKoqxzVBAoiqKsc1QQ1AAROV9EfiYi0yLy31d7Pkp9EZEfichvrPY8FKVWqCCoDf8D+KExptUY87HVnkwhInKHiDwvIjkReVvBtttF5HERmRKRQRH5iIj4XNsjIvJNEYmJyDEReVPB8W+yx2Mi8i0RiVR77HpFRLwi8iERGbJvHn4qIh2u7b8rIqft/8lnRCRQ5Bw3iogRkQ9V8X4RETkjIvcXjIdE5BMiMioikyJyn2vby0Tkh/b40YLjNorIv9vznxSRB0RkT8GxT4vIhIiM2d+BHtf2z4lISkRmXA+vvW3Avi73tj9exDW9XESeE5G4Pf9tBdtvFpEn7O/koIj8smvbi+3fQtz++2LXtu8WzCklIk8XHPsT+/MYLDbntYwKgtqwDdhfaqPzJV9FngR+C3iiyLYQ8D6gG9gDvBz4fdf2fwRSwCbgV4FPisjFAPbffwbeYm+PA5+o5tjFIBYr/l11C8Qa8+fANcDVQBvW5zdrv+crgQ9g/R+2AefY+7vn1QR8FHikyvf7G+DZIuN3ABHgQvvv77q2xYDPAO8vclwL8BhwpX3cvwL/KSIt9vZngFcaYzqArcAB4JMF5/iIMabF9cgWbO9wbfuLaq5JRLqBbwB/bM9rL/AV1/aLgC8Bfwi0A5cBj9vb/MC3gS8AnfY1fdsexxjzKvd8gQeBr7ne/kvAffb73gj8loj8QpF5r02MMfpYxgO4F8hi/ZBngPOAz2F98e/C+kHdDPwc8FNgCjgB/JnrHAOAAd5ubxsH3gVcBTwFTAAfL3jfX8P6IYwDdwPbqpjr/cDbKuzze8B37OdhrIX8PNf2fwP+2n7+l8CXXNt22Pu3Vjq2irn+CPgw8ACQAM7F+vF+GjgFnAQ+BHjt/Y8BV9rPf9X+PC+2X/868C37+W7gIfszPQV8HPC73tcA78FavI7YY68AngMm7f1/DPzGEr8vnfb3ZEeJ7V8C/tL1+uXA6YJ9PgB8xP6efajC+11jX+/bgftd4xfY38W2CsffDByt4rqmnM+/YDwA/BXwjGus5LxdvwXfEq7pHcCDrtdh+7tzgeuz/YsS57zF/k6Ja+w4cGuJOWaBAddYHLjI9fprwB8s5TuyGg/VCJaJMeYm4CfAbxvrbuEFe9ObsBayVqwFOAa8FejAEgrvFpHXFJxuD7AT+BXg/2LdudwMXAz8sojcCCAitwH/C3gtsMF+/3+v0SXdwJx2cx6QcV0TWNqFc1d/sf0aAGPMIezFv4pjq+EtWD/uVqyF/nNABksoXI7143Vs9T8GXmo/vxE4bF+L8/rH9vMs1p1vN9Yd+cuxtCU3r8H6X1zkusv8I/uYQ8C1zo4i0m+bQPqrvKZL7Wt4vW3+eUFE3uPaPu8ztZ9vEpEu+/22Yd0EfLDSG9ma6MeB38ZaXN3sxvpM/9w2DT0tIq+r8hoK3+fFgB846BrrF5EJrIX497EEl5vfEpGobYIp9r7HbBPLZ+3/QTXXVPh9jGH9v5zv3EvsczwtIqdE5AsyZ8q8GHjK2Ku4zVMU/76+FfiJMeaoa+z/Am8VkSYROR/ru3VPkWPXJCoI6se3jTEPGGNyxphZY8yPjDFP26+fwlq4byw45i/sff8LS3D8uzFmxBhzEmuxv9ze713AXxljnjXGZLDuzF9caA9dLCLya8Au4H/bQy1Yd3puJrEWZmf7ZIntlY6ths8ZY/bb1xgB/hvwPmNMzBgzAvw98AZ73x8z93lej3UX6rzOCwJjzOPGmIeNMRn7h/zPLPw//JUxJmqMSdjvud8Y83VjTBrrB3/a2dEYc9wY02GMOV7lNfViaTbnAduB1wN/JiKvsLcXfqbOc+dz+xjwx8aYmSre678DjxhjHi8xj0vs82/FWlj/VUQurPI6ABCRNixN78+NMfl5O58LlvD8IyyNyuFjWDc8G7HMOJ8TEUe4jmJpwtuwTE+twBervKZy30fnmt8CvM5+/yDwD1Ue6+atWDclbv4D63+ZwLrWTxtjHity7JpEBUH9OOF+ISJ7bOfVGRGZxFrMuwuOGXY9TxR57dhgtwEfte9EJ4AoIEAPS8TWTv4KeJUxZtQensGyYbtpA6ar2F7p2Gpwf4bbgCbglOu6/xlrMQFrob9eRLYAXuCrwLUiMoC18P7Mvs7zROQ/7LvxKSwhWvh/cL/vVvdr+47xBFVS4GDsx/o/AnzQGJOwbwq+jCVwYOHn5jyfFpFXA63GmK9QARHZirVo/mGJXRJAGstEkzLG/Bj4IZaWVe21BYHvAA8bY/6q2D7GmChz9nafPfaEMWbMFsZ3YS30r7W3zRhj9trbhrEE1C0i0lrFNVX6ziWAzxpjXrAF6V9S+nMvPNa55uuAzcDXXWMR4HtYWloz0Ae8UkQKNc01iwqC+lGotn4JuBPoM8a0A/+EtXgvhRPAO+07UecRNMY8uJSTicitwKeAVxtjnnZtegHwichO19hlzJmO9tuvnfOcg2UTfqGKY6vB/RmeAJJAt+ua24wxFwMYYw5i2Wl/B7jPGDOFdef+Diw7cs4+zyex7th2GmPasExshf8H9/uewvphO9co7tcVL2C+Q/Q4lrmh8D3cz+d9pvbzYWPMGJYZa5ctxE5jmRDfJyLfLvLWu4EtwDP2vh8FdtvHel3zmDfdaq9LrEimbwGDwDsr7O7DEtiFC637fUv9Fpw5eah8TYXfxzCW38r5zj1F+c/9Rfb/1+FFLPy+3g58o0AjOwfIGmM+bwuwQeYL97XPajspzoYHlmPzN1yvP0eBMwwYAW63n++2X3/Bfj1AgYMM6wf2UtfrLwB/ZD//RWAfc87QduCXyszPj3Wn8gDwm/Zzj73tJmAMuKHEsV/GMmOFsWzjk673vRjL/HO9vf0LwJerOXaxn6k99m2sH38b1sKwA7jRtf1L9nzeYr/+W/v1+137PAr8CdbCcwHwPPMdjgY41/W6G+uu8LVYC9p7sWz8S3IW2+e8D0ubCWBF7IwAL7e33YolwC7C8ifdy5xzvhXrbtR5fAXLPBYp8h6Bgn3fixVltNne3oRl0/9j+7quta/Tcax67O/Jq7B8Cc3YTnX72O9gCYIFTl37szrfPscGLO3sCdf212Nptx4sDWQa+7uO5Ztxju2yr/GHVV7TBvs79jp7vn+Dpa047/trwBGshTtkz+vfXL+RY/Y5A1iayDHmBxIE7fPfVHC9bVjBB2+y570Zy5n9l8X+/2vxseoTOBseVCcIXm9/saax7IkfZ4mCwH79FuBp5qKQPlNhfqbg8VJ72w+xFrYZ1+O7rmMj9g8+hhVF8aaCc7/JHo9hLdSRao7FEh4z1X6m9lg71h39oP2D/CnwBtf2d9rXts1+/fP26z2ufW7A0ghmsPwuH6SMILDHbsXScBZEDQH99rn6F/F96cEyJcxgObXfWbD997DMglPAZ4FAifPM+55hRUvtL7Hv29zXaY9djLVgxbBCPn/Rte2lRb4zP7K33Wi/jhd8b663t/8O1oIbwxJqX8YV1WZ/7pP29T1Z8D98o+vYU8DnsRf6Kq/pZvv/m7C/QwMF2/8cOGM//g3odG27HCucNIEVan15wbFvxPoNS5G53IQVUjtpX/OngFC91pxaP8S+CEVRFGWdoj4CRVGUdY4KAkVRlHWOCgJFUZR1jgoCRVGUdY4KAkVRlHVOvaorrhjd3d1mYGBgtaehKIqy5nj88cdHjTEbKu3X8IJgYGCAvXv3rvY0FEVR1hwicqya/dQ0pCiKss5RQaAoirLOUUGgKIqyzlFBoCiKss5RQaAoirLOUUGgKIqyzlFBoCiKss5RQaAoirLOUUGgKIqyzlFBoCiKsgT2D00yMjW72tOoCSoIFEWpC/c8M8z7vvzT1Z5G3XjbZx/j7+85sNrTqAkqCBRFqQt37z/Nt342RDqbW+2pVE0mm+PGv/0h33hisOx+sWSGM9NJhlUjUBRFKc2J8TgAU4n0Ks+kek5NznJsLM7eY+Nl9zs5kQBgLJZaiWnVHRUEiqLUhRNRa7GcbCBB4AivwfFE2f1O2tvHZpJ1n9NKoIJAUZSak87mODXZeILAEQCDtkAovZ+1PaoagaIoSnFOTcySM9bzhhIE0TmNIOdcQLH9bNNQPJUlkcquyNzqiQoCRVFqzgnXHXUjCYITtkaQyuQYLWP2OekyHY3FGt88pIJAUZSacyLaoIIgGsfrEet5GT/B4HgCezfGZhrfPKSCQFGUmnNifG5BnYw3jiAYHE9wSU+7/by0n+DkRIJzN7YAqhGURUT+VkSeE5GnROSbItLh2vYHInJQRJ4XkVe6xm+1xw6KyAfqNTdFUerLiWiCrR3NBJu8DaMRJDNZhqdneck5EaB05NBsOsuZ6SQv6rWWNNUIyvN94BJjzIuAF4A/ABCRi4A3ABcDtwKfEBGviHiBfwReBVwEvNHeV1GUBmNwPE5fZ4j2YFPDCIKT4wmMgZ0bW+luCZTUCE5NWklkl/VamsPZkEtQN0FgjPkvY0zGfvkw0Gs/vw34sjEmaYw5AhwEdtuPg8aYw8aYFPBle19FURqME+OJhhMEjgbQ1xmktzOYz4NYuJ8lIHZuaiXg85wVIaQr5SP4NeC79vMe4IRr26A9VmpcUZQGwjGd9EWCtIeamGgQQeBEOvVFQvR2BktqBE7EUG9nkO6WQNnookZhWYJARO4RkX1FHre59vlDIAN8cbmTdZ3zHSKyV0T2njlzplanVRSlBgy6FtT2YFPDlJg4EU3Q5BU2tTXTFwlxcqJ4LsHJiQRej7C5rZlI2H9WaAS+5RxsjLm53HYReRvw88DLjTHOJ3oS6HPt1muPUWa88H3vAO4A2LVrV+msD0VRVhzHpNLbGaQ92MS+BhEEg+NxtnYE8XqE3s4g6axheHqWLe3Bgv0SbG5rxuf10NXiV2dxOUTkVuB/AL9gjHHrWHcCbxCRgIhsB3YCjwKPATtFZLuI+LEcynfWa36KotSHvImlwXwEjl8DoNf+Wyxy6OR4gp5OSzicLRpBPX0EHwdage+LyM9E5J8AjDH7ga8CzwDfA95jjMnajuXfBu4GngW+au+rKEoDcSIaJ+DzsKE1QHuwiXgqSyqz9ktRD0bj9EWsBb7PXuiL+QlOTiTo7bC2Oz6COYNHY7Is01A5jDHnltn2YeDDRcbvAu6q15wUpVFJZrL4vR5EZLWnUpET0QS9nUFEhI5QE2BlF29oDazyzEoTS2YYi6XymsBWe6EvjBxyium5NYJkJkcslaUlULfltO5oZrGirHFiyQx7/vIHvPrj93Pvc8Nr/u7zxHicvoi1oLYH5wRBtRhjODgyzY+eH6nL/Irh9BfotRf45iYvG1sX5hKcnrSK6fXYgqIr7Acg2uB+gsYVYYqyTtg/NMVEPE02Z/i1z+3l8v4Ofu8V53Hdud111xA+c/8RJuIpfu+W86s+ZnA8wRX9nQC0VSkIUpkcjx2Ncs+zw/zg2RGO27WK/uN3rsuXfFguI1OzbGgNFP3MnNpIjgBznhf6COYEhrVfd4ul5YzGkvR3hWhUVCNQKmKM4Wt7TxBLZirvrNScfScnAfje+27gr157KcOTs7zl04/yK3c8PK+4Wz2488kh/t8TRYP3ijI1m2Yykc7fWTsaQbkQ0kcOj3HlX3yfX/2XR/jiI8fZsSHMB151gbXtSHQZs5/jzHSSa/76Xr7z1Kmi253P0Zm38/xEgUbgCAa3aQhqoxH8+6PH+cj3nlv2eZaCCgKlIofOxHj/15/iv545vdpTOevYd3KybN17gH1Dk2xoDdDTEeSNu/v54ftfygdvu5inByf5h3vr2zx9aCLB6alZMlX2HS68s3YEwUSi9EL52NEo08kM//yWK/nZn7yCz759N++6cQfbukI8cnhsmVdgMTSRIJMzJc1Ng+MJmps8bGiZ82P0dgY5NTH/2p1ksi3tzQB0tViCoBaF5+56+hTf/Gn1QreWqCBQKuK045uZVY2gkCeOj/OiP7ubkenFNzE/dGaGn/+H+/nOU0Nl99t/copLXeaRgM/LW68e4GUXbOAnB0br5jNIZrKMTCfJ5ky+vk4lHOeqE4bZ4ZiGylQgPTOdpLXZxysv3kzIP2et3j0Q4bGj0YqCshrG45YgerSEhnFiPE5vZ2ie2aivM0QmZzjtalB/ciLOxtYAzU1eALrCluCoRb2hM9NJxmZSq+IDUkGgVGTc/hHHzoJOTLXmuVPTTM1meOH0zKKPPTRiHfNwmbveRCrLgZFpLtnatmDb9Ts3cGpylkNnFv/e1TA8OXeX69jGKzGXVWyZTuZ8BKVvIkZnUvPuxB12b48wHk9zsAbX5wiCwfFE0WtxIp3cFMslGHTlEAAE/V5Cfm9NksrGYilS2RzTq2CCVUGgVGTC/hHF1UewAMcJOlTlQunGaXyy9+h4yX2ePT1FzsDFRRym1+/sBuDHL4wu+r2rYWhy/gJYDSeicVoDvrxJqMnrIewvX4r6zHSS7iKhpXu2dwG18RNEY3Pv/+iRhYLXqZbqpjefSzB37ScnEvmIIYdaJJXlciZ/jtXIVFZBoFTEKRo2k1SNoBBngav2jtmNc/d8YGQmL2wL2W87ii8tIgh6O0OcsyHMTw7Up96WW7idrFYQjCfojcw3sVTKLh6dSRbVCPoiQTa3NZc05yyG8VgKj0Brs2/B+SYTaaZmM3ktxmFrRxCROb9HLmc4NTGb1xQcumpQeG48niJrm8DGVqGInQoCpSKOWh1PqUZQyLI0gmgi38XriePFtYKnT04SCfvzzslCbti5gYcPjzGbrr2Qdq6pPdjEyYnqopNOROP5rFyH9pCfyTLO4jMzyaLJZiLC7u0RHj0ytmy7+Xg8RWfIz+6BCI8cni8I8g7uggXe7/Owua05rxGcmUmSyubmmYYAumugEbh9DKOqEShrkYmY+ghK4SxwbjNKtQyOx7n6nC58HilpHtp3coqLt7aVzBe44bxuZtM5Hj9W2ry0VIYmZ+kK+zlnQ7gq05AxhsHxxII75vagr6RGMJvOMj2boduOvilk9/YIw1PJfF7BUhmPp+gM+9m9PcLh0dg85/7g+PzcADfuctTO394ipqHlmnNGp+e0gNVofamCQKnIuPoISpI3DVVpOnEwxnAiGufcjS1cvLWNvUUW8mQmywvD00XNQg57tnfR5BXue6H25qGhiQRbOprp6QhWZfoai6VIpLMLTCzlTEOOSaVU+Yk92622kYV38YslGksRCfnZc47ld3jsyNznXejgdtPXOZdUVphD4NDVEmAstrx6Q2dc5iD1EShrkol81JAKgkLypqHJ2UWFOU7E08RSWfoiIa7cFuHJExMLCrM9f3qaTM6UzawNB3zs2hbhvgO1dxgPTSTY2h6kpzPIUIna/G5KmVjKCYIz9p1wdxEfAcC5G1uIhP3LdhiPx9J0hJq4eGsbIb+XR1wO40IHt5veziCnJhOks7m8MCx0FneF/aSzZlnRPo45yOsR9REoa5M5H4GahgpxFrhUJreoWPK5Us1Bdg10kszk2D80OW+ffSenALhka/kSC9ef182zp6aWlMtQjqGJWbZ2BOntDJHOGkamyy9QThSUu0wDVNIIrM+slEYgIuweiPDo0eUllo3HU0TCfpq8Hq7c1jnPYXzCDgktZn7r7QyRM1aNocHxBJ2hJsIFxeXySWXLuJMfm0ni8wh9nUFGV6GstQoCpSL5PAI1DS1gMp7O3yEuxmGcT7yKhNi1zarLU2jn3zc0SVuzr6jJws0NOzcAcH8NtYKp2TQzyQxbO5rzNvFKDuNiZRoAOkJ+ZtO5og7tShoBWH6CE9HEkhzyYJnhHB8BWOam505PMx5zcgviC4SXQ2/EqUIan9eHwE2+zMQybPujM0m6WvxsaA2oRqCsPYwxeYeoagTzyeYMU7MZLtxiJXstShCMzy2aG9ua6YsEFziM952c5JKe9oqF5S7a0kZX2F9TP4FzLZZGsDCevhiD43G6wv4Fd8xtZeoNOT6CrhLOYrAEAVilKJbCTDJDOmuIhPz2+bry57N8NYkF5iyHPldSmdWHYOF++cJzy9AIRmdSdLcE6AoH1EegrD1iqSzprGUbVo1gPtOz1sJ20ZZWYHG5BCeicTpCTbQ2W4vkVdsi7D0WzTsc09kcz52arqrypscjXLezm/sPjtakHAPMFwQ9VQqCE1Erh6CQcqWoz0wnaQ82EfB5S573wi1ttAZ8S/YTjNtRb05vhMv62vH7PDx6JJp3cBdqMQ6b25vxiCW4B8fjRTUCR4gtJ4R0bCZJV0vAan2ppiFlreGozxtbA8RT2TVfC38lcRa2/q4wIb+XoYnqbfSD4/PvQq8c6GR0JsWxMTvJbHiGVDZXdQnmG3ZuYHQmxTOnphZxBaVxrqWnI0jI7yMS9lcWBOPxogtqOUEwOpMsGTrq4PUIuwY6l1yAzvFxOSacgM/L5X0dPHIkmr+mUqahJq+HLe1BnhqcZDadW+Aodp93OSYdSyPw09USYDyeqrrIX61QQaCUxYkY6ukMkskZkg3QcnClcBa2jmATWzuCVSddgdO8ZW5R2bXNMn84YaRO6eliNYaK4ZSb+EmN/ARDEwl8HsmbPSqFkGZzhqGJ4iYWp/DcRJHCc6MlkskK2b29i0NnYkvK4I3agsDxEQDsOaeL/UOTPGsLznJ+mN7OIHtts1QxQRfweWkN+JZ8J2+MsZLqWgJ0t/gxZs4vt1KoIFDK4txNOXdC6ieYw1nY2kNN9HQEq9YIcjmzQCPYubGFtmYfjx+zFpx9Q5O0BHwMdIWrOufGtmYu2Nxas3ITQxMJNrc35zOfezqCnCzSv9fh9NQs6awpuqBWMg2VcxQ75P0ESzAPOVqt4yMAy2GcM/Atu+xzsWQyh97OUD6ZsphpCCDSsvSksplkhlQmR1eL31XNdGUdxioIlLLkBYH9A1A/wRzOwtZuawTVOovPzCRJZXLz7i49HuGKbZ15h/G+k5NctLUNj6f6DmQ3nLeBvUfHa1IKxAkddejttDSCUqbBwRI5BFDJNJSqSiO4tKedYJN3SX4Cx3bf6RIEV/R34vMIjxyJ0hlqKttv2C3cijmLwcolWOri7TiZu20fAax8UpkKAqUsedPQKmkE6Wxuxe2l1eIWBD0dzYzFUlXV/MmXKiiwS+/a1smBkRnGZpI8c2qqYv5AITfs3EAqm1t2Fi5YJTPc9vCeziCz6dK5EqVyCKB0u8pEKstMMlOVRuD3ebhiW8eSCtBNxNN4PUJr89xiH/R7eVFve8k5u3G0hZaAj7ZgcYERWUa0j2Pu6rZNQ+6xlUIFgVIWRxBsbbc1ghXMLr7/wCg3fOSHvPrjD3C6ysYoK0mhRgDVhZAWNm9xuNL2E3z98UFm0zku6anOP+Cwa6ATv8/DQ8vs6pXNGU5Pzs4rdFesNr+bE9E4IrC1Y2FxPK9HaA0srDdUqbxEIbsHunj29FTF/seFROMpOkNNC7Qrp9xEqdBRB0dz6y2RdAbQvYxonzFXCK1jGlpuEbvFooJAKct4PGWl39uhd/EVKEWdSGX5szv38+ZPP0KwycvxsRiv/cQDHBiervt7L4apRJqAz0Nzk9clCCoLrFKJVy/u68DnET7/0DGgeOnpcjQ3edneFebIaKzy3GfTJU1IZ6aTZHJmnmnI0Q5K1VR6YXia3s5gyTDQ9tDC7GInU7lYCepivLi/A2PguUVGRo3HUvPMQg6O36FU6KiDozEUixhyiIT9jMdSSwrfPeNkV7cEaA822WUmVBAoa4iJeIqOcBMhv/UDr7dG8OSJCX7uH37C5x48ytuvHeCu917PV955Nemc4XWffLAmtelrxUQ8nbd/91SZfQtWxNAGV7tDh6Dfy8Vb2zg5YfXPPWdDy6Ln1BcJVdXQ/u2ffYz/+f+eLrrNqaRaaBqC4tdnjOHRI1GusjWaYhQrM7FYjcBZsBdb6TVaQhBcNRChpyPIroHS8wbY3NaM3+spa0LqagmQyRmmZhcf7eNUHo2E/Xg8Qmdo6f6GpVJ3QSAi/5+IGBHptl+LiHxMRA6KyFMicoVr39tF5ID9uL3ec1MqMx5P0xnyE7Z7ydazJ8HnHjjCaz/5IIlUli/8+h7+9NUX09zk5ZKedr7x7mvobg3w5k8/wl1Pn6rbHBbDZGJOEGxub0YETlahEVgRQ8XvLh3z0EVb2vIRO4uhPxLieDReNt/DGMP+oUkeOFi837Fj3triMvO0B5tobfYVNQ0dOhNjLJbK32EXo5ggqKa8hBvHPLnYSq8T8TSd4YUF5VoCPh74wE284qJNZY/3eoRPv20X77zxnJL7dOeb2C/+Tn4slqQz1ITP68mfa6V7EtRVEIhIH3ALcNw1/Cpgp/14B/BJe98I8KfAHmA38Kci0lnP+SmVmYin6AjNlQ2I1ck0lMnm+Nu7n2f3QITvvfcGrrPj4h36IiH+37uu4dKedt7zpSf4ymPHS5ypPLmcqVkTF7cgaPJ62NTaXJ2PoExtm10D1ld+sWYhh/5IkHgqW3ZBGplOMpvOEY2lOFzEjOTOKnZjhZAuvD5HS1usIKimvISboN9LV9hflbB1E7ULzi2H63duYEt7edMQLC3aZ3Q6NU8YdrX4V7zeUL01gr8H/gfgvu24Dfi8sXgY6BCRLcArge8bY6LGmHHg+8CtdZ6fUgFLI2giHLBNQ0sIHz01mai4+D5zaopYKsub9vTn/RGFdIb9fPE39nBFfycfvefAkrKcP3bvAW7+ux+TrkEkklsQgOUorSQIMtkcQxOzJR268wZmAAAgAElEQVSUu7dHCPu9XHNud9Htlejvss5brpHLUdfiv7dI/Z6hiVlaAz7amuf/H3pdtfndPHpkjO6WANu7S+c8dBTxEYzOWHfCTd7ql6GtVfZGcDDGlPQR1JI5J+/iF3Aru9olCMKBFS8zUTdBICK3ASeNMU8WbOoBTrheD9pjpcaVVcRp8dfs8yKy+C5lsWSGW/7uPv7vPQfK7lfNXSVYDtHXXN7D0OQsR8cW37XqwYNjDI4nalKpczKRnie0qsklODU5SzZnSjoou1sC/PRPbuGWCuaKUvTbmkY5P4FTxqJUZ7ShicQCbQCK5xIYY3jkSJQ950TKFsdrCzYxGU/PO7baZDI3PYvI1wCYTmbI5Ez9BUE+7HMppqHUPK2oaxnJaUtlWYJARO4RkX1FHrcB/wv4k9pMc8H7vkNE9orI3jNn6tO4W7HuXqdnM7QHrdC7UJN30V3KfvT8GaaTGe59brjsfo8ciTLQFWJTW/HevG6u3WGF/T1wcHGLeTZn2GfX/P/Wz04u6thiFGoEPR3Big1qKtW2AStmvlLF0VI4YZ7HywjJo2MxfHahumItLocmE/P8A3PnDjKTzDCVmPsODI4nODU5m+8kVor2YBOpbI7Z9JwmVm0ymZuttnmqWm3QabPauUzTUCUcQbOUsM/RAoHY3RJgJpmpSx/qUixLEBhjbjbGXFL4AA4D24EnReQo0As8ISKbgZNAn+s0vfZYqfFi73uHMWaXMWbXhg0blnMJShkcVb7TvusNBXyL1gi+t/80AC8Mz5TMBcjlDI8djVbUBhy2d4fZ0t7Mg4cWJwgOn5khnsrS3eLnv/YPL8vxncnmmElmCkxDwYoNauYa0pSPXV8qzU1eNrUFypqGjkWt4nB7tndxeDS2wB5dmFXs4EQRnXCVmqhWkyuWXbwkjaAzSCKdLVq3qBjRfMG54ubGWuH3eWhr9i3atj+bzjKdzMwTiF3hpTuel0pdTEPGmKeNMRuNMQPGmAEsM88VxpjTwJ3AW+3ooZcAk8aYU8DdwC0i0mk7iW+xx5RVwil85dxNhf3eRS2es+ks9z47zIv7OgC4v8Qd/IGRGSbi6Xyd+EqICNfs6ObBQ2OLitt+atDSBt5383kk0lm+/0x5LaUcU7PW51CoEUD5ctSD0Tgeoegdd61wIodKcWwsxraucN4x7dYKZtNZorFU0Zj5uRDSuet79EiU9mAT521sLTunYoKg2oJz8+Zgf27V+gnGi5SXqBfdLYFFdxfLO8zDbtOQXW9oBR3Gq5FHcBeWxnAQ+BTwWwDGmCjwF8Bj9uOD9piySkzYd1Md9o8o5PctKmrogYOjxFJZ3nvzTrpbAiULoj1q94+tZF5wc93OLibi6UWVXX765CQhv5c3XNXH1vbmfMGxpZCvPFrgI4Dy2cUnxhNsaQ8uykG6WMrlEhhjODYaZ6ArxKU97fi9nnmCYC5iqJhpaGF28SNHxrhqIFKxJlJH0PoOOZ9bLJmxtbPFCgJrDtUKgmJ1hupFV4uf6CJt+2OuOkPu87i3rQQrIghszWDUfm6MMe8xxuwwxlxqjNnr2u8zxphz7cdnV2JuSmnyGoG92IUDi9MIvrfvNK3NPq7d0c31O7u5/0DxximPHImypb25Yoanm2t2WFE1i/ETPDU4wSVb2/F5PfzCi3u478Doku+63OUlHKppWTlYomZ/LemPhDg1NUsys1BoR2MpppMZtnWF7RyNtnmdv5zM6GKhkp2hJoJN3nwI6fCU5bCvRoC350tRW4vbYpPJHPJaSZW5BONFSlDXi8gSCs/l6wy5PofucGDetpVAM4uVkuR/RG6NoEofQTqb4/vPDnPzhZvw+zxcv7ObsdjCxilOVuru7eWjTgrZ1NbMuRtbeOBQdXV1Mtkc+4emuNQuNPaay7eSzRn+c4nJac6C5hYEbUEfYb+37N3qiWiiYpGz5dIfCWFM8cXSibTaZoeZ7hqIsO/kVN4x6QixYqYhEaG3M5gvmuf4B/acU70gcAToXDLZ4hbozlATzU2eqiOHxuMpvB6hrbl0ddFa0dUSWLSzeK7g3PyoITgLfATK2cGcacilEVQZNfTokSgT8TSvvHgzANedW7xxyrGxOCPTyaodxW6u3dHFY0eiRe98CzkwMkMyk8tXnLxgcxvnb2pdsnmomEYgImVDSJOZLMPTpXMIaoUTQlrMT3A8auUQbLP7HFy5rZNUNsfTdiOcockEIpSM3urpnIvjf/RIlLDfy0VbKhfHKxQES9UIRKRikxw30ZiVGb/UKKzF0BX2E11kvaHRIqahkN9Lc5PnrPcRKA3CeDyNzyP5Wu1hv6/qMtTf3XeKYJOXG8+zorpKNU551DZLLMY/4HDNud0k0ll+enyi4r5P245id8bubZdv5YnjE2VDLUvhNGJvC86PRunpLN2gxgp7rFzkbLmUyyU4OmpVCXVq7O/aZjmMnXyCoYkEG1sD+H3Fl4beAkFw5UAkXxqhHK3NPkTmPjd3obXFspjeD1YyWX0jhhy6wn5yBiYWUR11dCZJS8A3r+6UiKx4E3sVBEpJJuJpOkJN+bupcMDHTBUaQS5nuHv/MC89fwNB/9wX/Pqd3ew9Ok7CJUwePRIlEvazYwkF1l5yThcegQer8BM8dXKC1oKOX79w2VYA7nxy8VpBMY0Ayi9S5Wr215INrQECPk9RjeDYWIyt7XNVQrtaApzTHc53RhuamC1bSqGnI8REPM2JaJznh6erFuAej9DW3JRfJM9MJxFhSaUf3MKoEuPx1Ir4BwAiS4j2cXoVF9Ld4l90BNJyUEGglMSpM+QQqjJ89Inj45yZTnLrJZvnjV/vNE45MmfXf/RIlN0Di/MPOLQHm7i0t6MqP8HTg5Nc0tM+L7qltzPE7oEI3/rZ0KLLVUwm0gSbvAvKLvd0BBmLpeYJOwfHtl6uP24tEJGSIaRHx+IMdBf2Qejk8WPjGGMWNKQpxHHWfttOyFuMSc9db2h0Jkkk5K9Kmyhka3uQ0ZnqmgCNx1PzWlTWk+4lxP+PzSTz4aJuuloCahpS1gbj8flqdTjgI501pCo0sP/evtP4vR5uumDjvPHd2yP4fZ68n+DUZILj0ThXLcEs5HDtji6ePDFRVlNJZXI8e2o67x9w8wsv3srBkRn2Dy2uxr27BLUbJ+yyWKnkE9EETV5hU2v9cggcLEGwcA7Ho3H6I/NrAu0a6GQ8nubQmRm7vETp+TlmrW88cZKAz1P0My2FWxAsJZnMwRFG1ZiHorH0imkEc/H/1QsCq87Qwvl1hVe2zIQKAqUklmlovkYA5UtRG2P43v7TXHtuF60FRcuam7zs2R7J+wnyUSfLEQTndpPJmXwuQjFeGJ4mlc3lI4bc/NylW/B5JH+HWy2F5SUcnFLJxRapE+NxejqCi+pDvFScXAK3pjOZSBONpRjoKt4Z7b+eGWY2nStrGuq1tYXDozEu7+8o2YimGIUawWIdxQ7VNgEyxjARXzkfQb4C6SJCSC3TUAmNIJZcUmHFpaCCQCnJAo3A7klQLoR0/9AUg+MJXnXJlqLbrzu3O19u4tEjUVoCPi6sIuqkFFdus9ozPnCwtCBwMopf1NOxYFtn2M9Lz9/Ad548tagfXUlBUCaXYDBauvx0remPhJhJZvK5IDBXf2hb13yNYMeGMJ2hJr7zpBVKW6y8hEN3y5wjudpMcAd3lzJLI1janXq1TYCcgnPLLUFdLZ2hJkSq1wgy2Rzj8VRR01B3i5901uQz2OuNCgKlKMaYfFMah5BdirpcCOn39p3GI3BzieqZ1++0ooh+cuAMjx6Jsmugc0kNWByam7xcNdBZNrHs6ZMTtAebStrmr97Rzemp2XmLZiUKK486bG5vxlOiQc3geCKfnVtvioWQHh2zQkcLfQQiwpXbOnnWzvEo5yPweCS/fbGaXLurAulyNIJyn7GblSwvAeDzeugINlWtEUTjKYyBDcVMQ/ns4pXxE6ggUIqSSGdJZXLzTEPVaATf3XeKPdu7St6FXbC5le6WAHc+OcSBkZkl5Q8Ucs2Obp47PV0yE/OpwUle1Nte0iHtdAurpsWjQymNoMnrYVPbwr4EsWSGsViq7o5ih2J9CY7ZgqC/iFbibtdYzkcAlqDweYTL+xdqWOVwTENWZc3ckn0EzmdcKbvYSe5aKY0AqGpeDqPTC3MIHJz+BiuVVKaCQCnKREF5CXD5CEpoBDPJDIfOxBZ0F3Pj8QjX7+zOO4yX4x9wuNZOVnuwSPTQbDrL86eny3b8qqaZSyGlBAHMlUp242gsK6UROElrJ+YJgjgbWwOE/AuzbJ18goDPU3Hh/IXLtnL7NQNFz1OO9mATmZzJ90NYqkYA1eUSjBckRK4EL+7r4InjE1UllTmaQ/GoIdUIlDVAsR9Rvl1lCY3AUcUr/cCdLOOAz8OlRez2i+XSnnZam31F8wmeOz1NJmfKRrfkF83x6gRBOpsjnsqWFQRO1FAmm+P//NfzvPMLj7O9O5y/9noT9HvZ0BqYlyx3bCw+L4/CzSV2AbqtHcGKoby/fFUff/zzFy16Ts7ndejMDFB9r+JiVJNdPG73IlhJjWDXQITJRJoXRqYr7lusvISD89msVO9iFQRKURyNYJ5pKFC+gb0jPCrFbV9vawxX9HeWzGBdDF6PcMPODXznySFeGJ7/A3RKJ1zaW1rghAM+usL+qk1DxSqPutna0cypiVmOj8X55X9+iH+49yCvv6KX//id61Z0USrMJTg6FsvXGCqkucnLS3Z0cf6m8uWkl0OHIwhGLEGwXI3g1GSi7J33Shacc9htm9geK9L5rZC8aajI5+D4NVYqhFQFgVKUwoJzYPUjAErG7M/1Lyivim9sa+bt1w5w+zXbajFVAP7o5y8kFPDxG/+6N6+ZADw9OEFX2M/W9vJ2byvcsjrbbqmsYoeejiCpbI5bP3ofB4Zn+Ic3Xs7f/tJleUG6UvR1BvOCIJ7KMDKdZKBMX+F/evMV/P2vvLhu83E+r4O10Ag6g6SzhjNlTCfRWAqfR2hdwc+9LxJkY2uAx45UrqA/Gkvi93qKzs/v89AebFpSD+SloIJAKUphCWqwOpQBxEv0JHAW4I4qojT+9NUXc2uJENOlsKU9yD+/5UpOT87yW198It+c/ik7o7iSuaOvQjMXN462VFhnyMFp4n7B5lbueu/1vNouZbHS9EdCnJpMkMrk8nb5UhoBWNVl3SVBak1bXiOI4VlieQmHahrUjNuZ8StRcM5BRLhqe4S9R6sQBNNWeYlS8+tawTITKgiUokwUWdSDdmGsWAXT0EqF6xVyRX8nf/naS3no8Bgf+o9nSKSyHBiZqSr7tT9iOR8z2fJZ0zBXOK2URnDdud189Z1X89V3Xr1ieQPF6IuEyBkrpyEvCCKlNYJ643xeR0ZjRMKBZYUN5xvUlInQGY+l696ishi7ByIMTc7mS4qUYnQmWdQs5GBlF6+MRrCyuqrSMIzH04T93nk2fK9HCDZ5S1YgHY+nESm9QK4Er7+yl+dPT/Gpnxwhkc6SzZmyEUMOfZ0hMjnDqcnZiot3JdOQiNQkLHa5uHMJ8qGjZTSCeuP4VFLZ3JKTyRzypTzKaATReGpVbkqcFqB7j46XjRIbnUmysawgCOQd6/VGNQKlKBOJVFETTzjgJVbKRxBL0R5sWtadXi34wKsu5MbzNvDVvYMAvKiMo9ghX7q5isihvLN4FQVeNbjDYo+OxYmE/asqpFsCvvx3YzmOYoDW5ibamn3lTUOx1Io65x0u2NxGa8CXL7FeirES5SUculr8mkegrC4T8XRRp2+oTE+C8VW6AyvE6xE+9sbLOWdDmC3tzWxqq7zo9JWp4V/IZIleBGuNTa3N+L0eTtgaQTn/wEogMtcpbCl9CAoplq/hZjxe/Gam3ng9whXbOss6jI0xjMUqmIZaAozHU1WZK5eLCgKlKOPxVL7huJuQv7RGMBFPr1iBr0q0B5v4+ruu4Yu/sacqZ+GW9ma8HqnKYTxhm83q2YC+Fng8Qm8kaJuGSucQrCSORrJcjQDK9yVwSqSsho8ArEq7B0Zm5kWwuZlMpElnDV1lNJbuFj/GsKjSJ0tlbX+TlVXDaUpTSDhQWiOIxtaGRuAQCfs5p8qGNz6vh60dzVWFkJbLKl5r9EdCHByZYWgyseoaAcwJguWEjjpsLZNUNjWbIZszq/Z9zHd+O1Y8n8BJFCsnEOfKTNTfYayCQClKKTNPOOArGTVU2Mim0SjVzKWQyUR6zZuFHPojIQ6MzGBM+dDRlaLd/n7UQiPo6QgyPZthanbhHfP4KtQZcnNZXwd+r6dkGOlcVnF5HwGsTFKZCgJlAdmcYTJR3MwT9ntL5xGsoipeC/o6QxVD/sAKH13J+jXLwV1grrD89GpQa40AikcORVc5lLm5ycuLettLOowdQdBVJnrKiawqVUyxlqggUBYwlUhjTPHEsJC/eN/i2XSWRDrb0BpBXyTE6EyqpA/EoZFMQ+5Q2LXhI7CdxbXQCMp0KptYhfIShewaiPD04GTRtqXOXX5ZjcAxDTW6RiAivyMiz4nIfhH5iGv8D0TkoIg8LyKvdI3fao8dFJEP1HNuSmmcBuPFoobCgeJ9i1c7mawWOIvmYIUywhOJVMMIAkcjaG32rQlH/pxGsPzvSb5BTZH/V9QpOLeK38fd2zvJ5Aw/OzGxYNvoTBKPlP+9OKHYK+EjqFtCmYi8DLgNuMwYkxSRjfb4RcAbgIuBrcA9InKefdg/Aq8ABoHHROROY8wz9ZqjUpy5yqPFNYJi1UfnKj2u/mKzVNwJWOdvLl18rRE1goGu8IqWWijFS8/fyNDEbE1uGDa0BGjyStEGNfmmNKv4fbyyP4IIPHY0ytU75ndzG51JVsyu9niEyAr1Lq5nZvG7gb82xiQBjDEj9vhtwJft8SMichDYbW87aIw5DCAiX7b3VUGwwkyUubsP+72kMjnS2dy88MlywqNRqKZBTTKTZTadaxhB0BLwsaE1kK9/tNpcNRDhqoHaZF17PMKW9uKRQ9G4VXCuZYUL/blpDzVx/qZWHivwEyQzWZ47PV2VVtQV9q9IKep6mobOA64XkUdE5McicpU93gOccO03aI+VGldWGOfuvljmbL7wXIFWcDaYhiJhP2G/t2zkUL68RANd5z+9+Qp+/5bzV3sadaGnRIOaiXiKzvDKFpwrxlUDEZ44Np5PCjs+Fud1n3yQnx6f4PVX9lY8vttuYl9vliUuReQeYHORTX9onzsCvAS4CviqiJyznPdzve87gHcA9Pf31+KUiotyi7pTijqeysy7K662BPVaRkToi5SPHKpUcG4tcuW21a97VC+2dgSL9quOxlKr6h9w2DXQyb89fIznTk8zOJ7g/V9/Eo8I//LWXSX7ervpavFz/Hj1nfOWyrIEgTHm5lLbROTdwDeMMQZ4VERyQDdwEuhz7dprj1FmvPB97wDuANi1a1flnnDKopiIp/GI5WAsxNEIYgUhpPkS1EWykRuJvkgoX6CtGE4J6kYSBGczPZ1BhqdnF5oqY8VLpKw0TvHB93/9KZ49NcVlve18/E1XVF2V9oadG9hcoZdGLainaehbwMsAbGewHxgF7gTeICIBEdkO7AQeBR4DdorIdhHxYzmU76zj/JQSODVaPEUcWW6NoPCYloCvJh3HVpO+TqtBjXX/spBKlUeVlaWnoxlj4PTkfIfxalUeLWRLe5CejiDPnpribdcM8LV3XbOo0uSvu7KXP3jVhXWcoUU9PSmfAT4jIvuAFHC7rR3sF5GvYjmBM8B7jDFZABH5beBuwAt8xhizv47zU0owUSZhKlxCIyhVpK7R6I8ESaSzjM6kisa6N0rl0fWCkyT3d99/gQ/edjGtzdb/xfERrAX+zy9fRjKT48bzNqz2VEpSN0FgjEkBby6x7cPAh4uM3wXcVa85KdUxUeZuKux3BMF8jWCt1RlaKn2uENJygkA1grXBnu0R3vOyHXzyR4d47GiU//1Ll7F7IGJlua+R7+NLzumqvNMq09h6vFIXxmOlq4iGAsW7lDV6nSGH/nxSWXEHXaU2lcrKIiK8/5UX8LV3XY3XI7zxUw/zR9/eZxWcWyMaQSOggkBZQLlF3dEIFoaPpomsgczV5eJ0lDo+VlwQTCbStLoarChrgyu3Rbjrv1/Pr+7p50uPHAdYE5nUjYK2qlQWMB5Pl7SB5zWCAtPQeOzs0AiCfi8bWgMlO5VNNVDl0fVGOODjQ6+5lFdctJlP3Xd4TbQLbRRUECjzcIrHlVKrQ01O1NCcRpDO5phOZs4KHwGUL0c92UCVR9crN563YU07ZtciahpS5uHYwEstdj6vh4DPM89H4CSgNXKdITd9ncGSDWoaqc6QolSLCgJlHk46e7m7+3DAN68nwZzwOHs0glOTCdJFesVOqCBQzkJUECjz2HvUaq13QZnqmyG/d75GEGv8OkNueiMhcqZ4nXvVCJSzERUEyjzufW6Ec7rDZXv9thRoBPnaRGeJachdjroQFQTK2YgKAiVPLJnhoUNjvOyCjWX3W6AROAXnzhKNwEkqK/QTzKazpDI52tVZrJxlqCBQ8jxwcJRUNsfLKwiCcMA3L3z0bChB7WZzWzNNXlmgEWhWsXK2ooJAyXPvcyO0BnzsqtA4JOT3zgsfHY+laG7yELQL0jU6Xo/Q0xFckEuglUeVsxUVBAoAuZzh3udGuOG8DRUriIb9vgWmobNFG3Doi4QWdCpTjUA5W1FBoACwf2iKkekkN1UwC4GVXTw/fPTsyCp2Y/UliHNwZJrRmSSZbE4FgXLWopnFCmCZhUTgpedXzsgs1AisyqNn1+J47oYWJhNpbv67+/JjjqakgkA521BBoABw73PDXN7XQVfLwtLLhYT8PmbTObI5g9cjTMTTXLg1uAKzXDne/JJtnLeplbFYkol4mol4mvF4ipDfS19n9Y1FFKURUEGgMDI9y5ODk/z+LedVtX84MNelrLW5ifH42acR+H0ertvZvdrTUJQVQX0ECj967gwAN11QuZk2WBoBWIXnsjnDROLscxYrynpCNQKFHzw3zJb2Zi7cUrqshJuwqxS13+vBmLMnh0BR1iOqEaxzkpksPzkwyk0XbESkumYr7uY0Z1t5CUVZj6hGsM559EiUeCrLyy+sHDbq4DSnmUlmSGasMNKzLXxUUdYTKgjWOT94doTmJg/X7KjeMTqnEWTI2ZWa10qjcEVRFo+ahtYoI1OzPHJ4rK7vYYzhB88Nc82Obpqbqi8PMecjyJ51dYYUZT2igmCN8okfHeL2zz6KMaZu73HozAwnoomqsondhFwagSMIOtRHoCgNiwqCNcrh0Riz6Vy+xHM9uO+FUYBF93d1TEOWRpDG5xFaA2plVJRGRQXBGuXYWAywkr3qxYOHRtnWFcrX368Wp8poPJXJ1xmqNuJIUZS1R90EgYi8WEQeFpGficheEdltj4uIfExEDorIUyJyheuY20XkgP24vV5zW+tksjlOjltNUYanknV7j0cORxflJHbw+zz4vR5iqexZWWdIUdYb9dTnPwL8uTHmuyLy3+zXLwVeBey0H3uATwJ7RCQC/CmwCzDA4yJypzFmvI5zXJMMTcySyVm+gZGp+mgET52cZDqZ4bpzl1ZGIRzwEk9mrBLUYXUUK0ojU0/TkAHa7OftwJD9/Dbg88biYaBDRLYArwS+b4yJ2ov/94Fb6zi/NcuxaCz/fGS6PhrBAwcs/8DVO7qWdHzI7yOWyjJxFtYZUpT1Rj01gvcBd4vI/8YSONfY4z3ACdd+g/ZYqfEFiMg7gHcA9Pf313bWa4BjY1ZDFI/UTyN44NAoF21pI7LEu/lwwEs8lSEaS3NFv2oEitLILEsjEJF7RGRfkcdtwLuB3zXG9AG/C3y6FhMGMMbcYYzZZYzZtWHD4iJeGoHj0Th+n4ft3eG6+AgSqSxPHJvg2nOXpg2ApRHMJG2NQE1DitLQLEsjMMbcXGqbiHweeK/98mvAv9jPTwJ9rl177bGTWD4E9/iPljO/RuXYWIz+SIhNbYG6RA3tPRYllc1x7RL9A2BpBCNTli9DTUOK0tjU00cwBNxoP78JOGA/vxN4qx099BJg0hhzCrgbuEVEOkWkE7jFHlt3HBuLsy0SYmNrc100gvsPjtLkFXZvL9+kvhwhv49BO7JJ6wwpSmNTTx/BbwIfFREfMItt0wfuAv4bcBCIA28HMMZEReQvgMfs/T5ojInWcX5rEmMMx6Nxrt7Rhd/n4cx0EmNMTeP0Hzw4xuV9nfkM4aUQ9nuZSVrtKrXOkKI0NnUTBMaY+4Eri4wb4D0ljvkM8Jl6zakRODOTJJ7Ksi0SImsglc0xUcMQzYl4in1Dk7z35TuXdZ6QK5NYS1ArSmOjmcVrjON2xNC27jAbW63+wbUMIX3o0BjGsOT8AYewf65InZqGFKWxUUGwxnBCR7dFQmxqawZguIYhpA8cGiXs93JZX8eyzuM2K6lpSFEaG60UtsY4Fo3jEejtDOGx/QK11AgePDjG7u0RmrzLuwdosU1DItAWVNOQojQyqhGsMY6NxdjSHsTv87CxzTIN1UojGJpIcHg0tqywUQenS1l7sAmvRwvOKUojo4JgjXFsLM62LqsaaMjvozXg40yNNIIHDlplJWohCJxS1GoWUpTGRwXBGuN4dE4QAGxsC9RMI3jw0BhdYT/nb2pd9rlCtrO4Q5PJFKXhUUGwhpieTRONpeiPhPNjG1uba+IjMMbwwMFRrt7RhacGppyw7SPQFpWK0vios3gN4UQMDbg0gk1tAfYeW3wl7mNjMU5PzhKNpYjGUwyOJxiZTtbELARujUAFgaI0OioI1hDHo5Yg6J9nGrI0gsVkF39t7wne//WnFoxvbA3wsvMX15+4FI5GENFkMkVpeFQQrCGO2u0pt3W5TUMBUpkck4l0VXffmWyOj/7gAJf2tPM/b72ASNhPJOynM9xEwPDaMLQAABc6SURBVOeteHy1qEagKGcPKgjWEMfH4nSF/fkYfbA0ArByCapZdL/z1BCD4wn+7NUXc93O2piBitHdEuCiLW1c0d9Zt/dQFGVlUEGwhjg2Fp9nFgLY1DqXS3BehWifXM7wiR8e4oLNrdx0QW1MQKVobvJy13uvr+t7KIqyMmjU0BrieNQqP+0mrxFUUY76+88Oc2Bkhne/dEdNIoMURVkfqCBYIyQzWYYmE/S7/ANA1YXnjDF84ocH2dYV4ucu3VK3eSqKcvahgmCNMDiewJj5oaNgRee0BHwVk8oeODjGk4OTvOvGHfiWWUdIUZT1ha4Ya4Rj+Yih0IJtG9sCFctM/OMPD7KpLcBrr+ipy/wURTl7UUGwRnCSydxZxQ4bW8uXmXj82DgPHR7jN68/p6YhooqirA9UEKwRjo3FCfm9dLcsDBHd1Fa+zMQnf3SQzlATb9rTX88pKopylqKCYI1wPBqnPxIqmj3saARWl8/5PHtqinueHeHXrt2+rB7EiqKsX1QQrBGOjcWK+gfA0giSmRxTs5kF2779syGavMJbrx6o8wwVRTlbUUGwBsjlDCfGEwx0LfQPAGxwQkiL+AkeOjzGZb0dtGs5aEVRlogKgjXA6alZUpncgqxih02uMhNupmfT7Ds5ydU7uuo+R0VRzl5UEKwB8sXmikQMwVxSWWHk0GNHo2RzRgWBoijLQgXBGuC4HTpaykewsYRG8NChMfw+jxZ+UxRlWSxLEIjIL4nIfhHJiciugm1/ICIHReR5EXmla/xWe+ygiHzANb5dRB6xx78iIuumvvGxaByfR9jS3lx0e0vAR9jvXaARPHR4jCv6O2hu0twBRVGWznI1gn3Aa4H73IMichHwBuBi4FbgEyLiFREv8I/Aq4CLgDfa+wL8DfD3xphzgXHg15c5t4bh8JkZ+iOhsqUhCnMJJuNp9g9NcfU59Ss1rSjK+mBZgsAY86wx5vkim24DvmyMSRpjjgAHgd3246Ax5rAxJgV8GbhNrOD5m4Cv28f/K/Ca5cytkTgwPMPOTS1l99nQGpgXNfTIkTGMQf0DiqIsm3r5CHqAE67Xg/ZYqfEuYMIYkykYP+uZTWc5Ohar2GugUCN48NAYzU0eLutrr/cUFUU5y6mYiioi9wCbi2z6Q2PMt2s/pcqIyDuAdwD09zd2WYXDZ2LkDOysIAjc2cUiwsOHx9i1LaK1hRRFWTYVBYEx5uYlnPck0Od63WuPUWJ8DOgQEZ+tFbj3LzanO4A7AHbt2rWw7kIDcWBkGoDzKpiGNrU1M5vOMZ3MkM7keO70NO9/5daVmKKiKGc59TIN3Qm8QUQCIrId2Ak8CjwG7LQjhPxYDuU7jVVE54fA6+3jbwdWRdtYaV4YnsbrEbZ3F88hcNjYNpdd/MiRKAAvOUf9A4qiLJ/lho/+oogMAlcD/ykidwMYY/YDXwWeAb4HvMcYk7Xv9n8buBt4FviqvS/A/wR+T0QOYvkMPr2cuTUKLwzPsL07XNHEs7F1rmXlQ4fGCPm9vKhX/QOKoiyfZZWrNMZ8E/hmiW0fBj5cZPwu4K4i44exoorWFQeGp7loa1vF/RyNYHh6locOj3HVQIQm7USmKEoN0JVkFZlNZzkWjbNzY3lHMcyVmXhmaIqDIzMaNqooSs1QQbCKHByZwRgqho6ClV0c8nv5zpOnALha/QOKotQIFQSrSLURQwAiwsbWAKenZmkN+Li4CnOSoihKNaggWEVeGJ6hySsMVIgYcnCKz+05J1K2HIWiKMpi0NVkFTkwPM327nDVTl/HT6Bho4qi1BIVBKvIC8MzFTOK3TgNatRRrChKLVm33c5n01mAVSvhnEhlOTEe53VX9FZ9zMvO38ipyQQXblb/gKIotWNdagSpTI4L/vh7fOq+w6s2h7mIocqOYofrdnbziV+9Eo9H6jgzRVHWG+tSEPh9HgI+D9PJTOWd68QLw1bE0GJMQ4qiKPVgXQoCgLZgE1OJ9Kq9/wsj0/i9HgZKtKdUFEVZKdatIGht9jE9u3oawYHhGc7ZENYwUEVRVp11uwq1NTcxNbuKGsHwtJqFFEVZE6xbQdDa7GNqlTSCWDLD4HiC8zZW7yhWFEWpF+tWELQFm5heJR/BwZEZQB3FiqKsDdavIFhFjcCJGFpM6KiiKEq9WMeCYPV8BAdGZvD7PGzrqq7GkKIoSj1Zt4KgtdlHKpPLZxivJC8MT7NjQwteTQxTFGUNsG4FQVuwCWBVQkgPDM+oWUhRlDXDuhUErc1WmaXpFTYPzSQznJxIVNWMRlEUZSVYt4KgrdnSCFbaYXzAKS2hoaOKoqwR1q0gaG12TEMrqxEcGLZCR1UjUBRlrbBuy1C3Ba1Ln0rUTyMwxjARTzMynWR4apaR6SR3PjlEwOehL6I1hhRFWRusW0GwEhrBH31rH1985PiC8Vsu2qQRQ4qirBnWrSBos53F9cwl+OnxCS7a0sa7X7qDTW3NbGoLsLG1maB/dZrhKIqiFGNZPgIR+SUR2S8iORHZ5Rp/hYg8LiJP239vcm270h4/KCIfExGxxyMi8n0ROWD/7VzO3CoR9vsQqW/46OmpWS7v7+DVl21l9/YI27rCKgQURVlzLNdZvA94LXBfwfgo8GpjzKXA7cC/ubZ9EvhNYKf9uNUe/wDwA2PMTuAH9uu64fEIrQFf3XoSzKazRGMptrQ31+X8iqIotWJZgsAY86wx5vki4z81xgzZL/cDQREJiMgWoM0Y87AxxgCfB15j73cb8K/28391jdeN1uamumkEw1OzAGxuD9bl/IqiKLViJcJHXwc8YYxJAj3AoGvboD0GsMkYc8p+fhrYVO+JtQXrV2/o1KQlCFQjUBRlrVPRWSwi9wCbi2z6Q2PMtyscezHwN8Ati5mUMcaIiClz3ncA7wDo7+9fzKnnUc+eBKcnHY1ABYGiKGubioLAGHPzUk4sIr3AN4G3GmMO2cMngV7Xbr32GMCwiGwxxpyyTUgjZeZ0B3AHwK5du0oKjEq0NTcxOB5f6uFlcTSCzW0qCBRFWdvUxTQkIh3AfwIfMMY84Izbpp8pEXmJHS30VsDRKu7Ecixj/y2rbdSCtjr2LT49maCt2Uc4sG4jdBVFaRCWGz76iyIyCFwN/KeI3G1v+m3gXOBPRORn9mOjve23gH8BDgKHgO/a438NvEJEDgA326/rSr19BFvUUawoSgOwrNtVY8w3scw/heMfAj5U4pi9wCVFxseAly9nPoultdnHTDJDLmfw1DjT9/TUrPoHFEVpCNZt0TmwfATGwEyq9uYhSyNQQaAoytpnXQuCuZ4EtRUEqUyO0ZmkagSKojQE61oQOF3Kap1dPDI9izGaQ6AoSmOwrgVBvTSCuRwCdRYrirL2WdeCoK1Opag1q1hRlEZiXQuC1jqVotasYkVRGol1LQgcH0GtTUOnJmcJ+720ajKZoigNwLoWBHmNoMbO4tNTCTa3N2O3WlAURVnTrGtBEPB5Cfg8ddEINKtYUZRGYV0LArB6EtTDR6D+AUVRGoV1LwjagrUtRZ3J5hiZTmrEkKIoDcO6FwStzU019RGMzqTI5oxqBIqiNAzrXhDUuhT1qckEoDkEiqI0DioIauwjyOcQtKmzWFGUxkAFQbDWGoFmFSuK0lise0FQax/B6alZAj4PHaGmmp1TURSlnqx7QdDW7COZyZHMZGtyPqcPgSaTKYrSKKx7QdDaXNsyE6cnExoxpChKQ7HuBUFbcPGlqD/xo4N844nBots0q1hRlEZj3VdFaw0svjnNZx84Ssjv5Rcv75lnAsrlDMPaq1hRlAZDNYJFViB12lAeG4tzeDQ2b9tYLEU6azRiSFGUhmLdC4LF9iQYnrLaUALc++zIvG1zOQQqCBRFaRzWvSCY0wiqEwRDE1bmsNcj3PPs8Lxtc1nF6iNQFKVxWPeCYK4nQXWmISdh7BUXbmLvsXEm43MC5PSUdiZTFKXxWJYgEJFfEpH9IpITkV1FtveLyIyI/L5r7FYReV5EDorIB1zj20XkEXv8KyLiX87cqqXF70NkERqBfdf/5pdsI5sz/PjAmfy2U5OzNHmFrvCKTF1RFKUmLFcj2Ae89v9v7/6D6yrrPI6/P8ltfpAmDW1DtrRI+VGB6ipitDCLLhZW8ccIOKyK7m7/6MrujLujozMI48yO/rMz7h/+2F3HWRCVHVZdRREG0QoVh3+WH0GKFrKFKiz9kZKA3CAloU3y3T/Oc8ttm9570yS9Ofd+XjN37j3PObd8v/Q03zzPc55zgPuPsf/LwM9KG5Jaga8D7wXWA9dIWp92fwn4SkScDbwIbJ5jbDVpaRFL22u/FfVwcYJlnUu46KwVLO9q45dlw0P7xibo7+mgpcWLycwsP+ZUCCJiKCJ2zLRP0pXA08DjZc1vB3ZGxO8j4gDwfeAKZddgbgRuS8fdAlw5l9hmYzY3nttbHGfVsg5aW8Ql5/TxqydHmZyaBrI5Al8xZGZ5syBzBJKWAp8DvnjErtXArrLt3altBVCMiMkj2k+I7o5CzXMEe8cmOLU3mwy+7Lx+iq8c5NFdRaD0ZDJPFJtZvlQtBJLulbR9htcVFb72BbJhnpfnLdLDY7pW0qCkwdHR0epfqKKnc0nNcwTDY+Oc2pv91v+OdSsptIitQyNExKH7DJmZ5UnVlcURcdlx/LkbgKsl/QvQC0xLmgAeAU4rO24NsAd4AeiVVEi9glL7sWK6EbgRYGBgII4jvsP0dBTYU5yoetz4gSmKrxw8dHlod8cSNpy5nK1Dz/F37zyTVyenvYbAzHJnQYaGIuIdEbE2ItYCXwX+OSL+HXgYWJeuEGoDPgrcGREB3Adcnf6ITcAdCxHbTHo6ausRlK4YKvUIADae289TIy/z4NN/APwcAjPLn7lePnqVpN3ARcBPJW2pdHz6bf8fgC3AEPCDiChNJn8O+IyknWRzBjfPJbbZyOYIqheC4WLpoTOvzQNcdt4pAHz3oWcBryEws/yZ003nIuJ24PYqx3zhiO27gbtnOO73ZFcVnXA9nUt4+dVJpqej4qWfh3oEZYXg9BVdnNXXxf1PZnMVXlVsZnnT9CuLIesRTAfsP1D5yqFSj6B/Wfth7Zee1w9kt53o624/6ntmZouZCwHZHAFUvwPp8Ng4K5e2015oPax947nZ8NAp3e20ejGZmeWMCwGvPaWs2qKybA3B0XMAA6efTE9HwfMDZpZLTf9gGqj9KWV7i+Oc1dd1VHuhtYXPv/88Otv8v9PM8sc/uSjrEVS4cigiGC6Oc/HZK2fc/5G3vW5BYjMzW2geGiJbUAaVewQvTUyy/8AUq3t9VZCZNRYXAmqbIzj00JkZ5gjMzPLMhYDXHk5TqUcw02IyM7NG4EIAdCxppa3QUnGOYKbbS5iZNQIXgiR7JkHlHkFrizil24XAzBqLC0HS01GoOEewd2ycfi8YM7MG5EKQdHcuqTpHsMpXDJlZA3IhSHqq3IF0rx9DaWYNyoUgqfRMgtLTx051j8DMGpALQdLdUTjm0NAL+w9wYHLaPQIza0guBElP55JjThaX1hC4R2BmjciFIOluLzBxcJoDk9NH7ZvpgTRmZo3ChSDp6Sw9k+DoXsFw0beXMLPG5UKQVLrNxPDYBG2FFlZ0tZ3osMzMFpwLQdJT4cZze8cmWLWsA8mLycys8bgQJBV7BEWvITCzxuVCkJTmCGZaVDY8NuGJYjNrWC4EybF6BFPTwb6XJjxRbGYNy4UgOdQjOGKOYOSPE0xNh59DYGYNa06FQNJfSnpc0rSkgSP2vUnS/6T9v5XUkdrfmrZ3SvpXpRlYScsl3SPpqfR+8lxim62lbQWko4eG9qbFZH5EpZk1qrn2CLYDHwLuL2+UVABuBf4+It4AXAKUfsJ+A/gEsC69Lk/t1wNbI2IdsDVtnzAtLeKMFV187+Fd/G705UPtfkSlmTW6ORWCiBiKiB0z7Ho38JuIeCwd90JETElaBfRExAMREcB/Alem71wB3JI+31LWfsL8x1+/lenp4GM3PcAzz+8H/IhKM2t8CzVH8HogJG2R9GtJ16X21cDusuN2pzaA/ogYTp/3Af0LFNsxrevv5rufuJCDU8E1Nz3Asy+8wt6xcbraWulJk8lmZo2maiGQdK+k7TO8rqjwtQJwMfDx9H6VpEtrDSr1FqJCTNdKGpQ0ODo6WusfW5Nz/qSbWzdvYPzgFNfc9ADbdhVZ1dvpxWRm1rCqFoKIuCwi3jjD644KX9sN3B8Rz0fEK8DdwAXAHmBN2XFrUhvAc2noiPQ+UiGmGyNiICIG+vr6qqUwa+tP7eHWzRv448RBHn226MVkZtbQFmpoaAvwp5JOShPHfw48kYZ+XpJ0Ybpa6G+AUkG5E9iUPm8qa6+LN65exq1/u4HujgJnn7K0nqGYmS0oZaMwx/ll6Srg34A+oAhsi4j3pH1/BdxANsRzd0Rcl9oHgO8AncDPgH+MiJC0AvgB8Drg/4APR8QfqsUwMDAQg4ODx51DNS/uP8BJ7a20F1oX7L9hZrYQJD0SEQNVj5tLIVgMFroQmJnlVa2FwCuLzcyanAuBmVmTcyEwM2tyLgRmZk3OhcDMrMm5EJiZNTkXAjOzJudCYGbW5FwIzMyanAuBmVmTy/0tJiSNkt2b6HisBJ6fx3DqpVHygMbJpVHyAOeyGNWax+kRUfUWzbkvBHMhabCW+3Asdo2SBzROLo2SBziXxWi+8/DQkJlZk3MhMDNrcs1eCG6sdwDzpFHygMbJpVHyAOeyGM1rHk09R2BmZu4RmJk1vaYsBJIul7RD0k5J19c7ntmQ9C1JI5K2l7Utl3SPpKfS+8n1jLEWkk6TdJ+kJyQ9LulTqT2PuXRIekjSYymXL6b2MyQ9mM6z/5bUVu9YayGpVdKjku5K23nN4xlJv5W0TdJgasvd+QUgqVfSbZL+V9KQpIvmM5emKwSSWoGvA+8F1gPXSFpf36hm5TvA5Ue0XQ9sjYh1wNa0vdhNAp+NiPXAhcAn099DHnN5FdgYEW8Gzgcul3Qh8CXgKxFxNvAisLmOMc7Gp4Chsu285gHwrog4v+xSyzyeXwBfA34eEecCbyb7+5m/XCKiqV7ARcCWsu0bgBvqHdcsc1gLbC/b3gGsSp9XATvqHeNx5HQH8Bd5zwU4Cfg1sIFswU8htR923i3WF7Am/VDZCNwFKI95pFifAVYe0Za78wtYBjxNmtNdiFyarkcArAZ2lW3vTm151h8Rw+nzPqC/nsHMlqS1wFuAB8lpLmk4ZRswAtwD/A4oRsRkOiQv59lXgeuA6bS9gnzmARDALyQ9Iuna1JbH8+sMYBT4dhqy+6akLuYxl2YsBA0tsl8PcnMpmKSlwI+AT0fES+X78pRLRExFxPlkv1G/HTi3ziHNmqQPACMR8Ui9Y5knF0fEBWTDwJ+U9M7ynTk6vwrABcA3IuItwH6OGAaaay7NWAj2AKeVba9JbXn2nKRVAOl9pM7x1ETSErIi8F8R8ePUnMtcSiKiCNxHNoTSK6mQduXhPPsz4IOSngG+TzY89DXylwcAEbEnvY8At5MV6DyeX7uB3RHxYNq+jawwzFsuzVgIHgbWpSsh2oCPAnfWOaa5uhPYlD5vIhtvX9QkCbgZGIqIL5ftymMufZJ60+dOsrmOIbKCcHU6bNHnEhE3RMSaiFhL9u/ilxHxcXKWB4CkLkndpc/Au4Ht5PD8ioh9wC5J56SmS4EnmM9c6j0RUqfJl/cBT5KN436+3vHMMvbvAcPAQbLfFDaTjeNuBZ4C7gWW1zvOGvK4mKwr+xtgW3q9L6e5vAl4NOWyHfin1H4m8BCwE/gh0F7vWGeR0yXAXXnNI8X8WHo9Xvp3nsfzK8V9PjCYzrGfACfPZy5eWWxm1uSacWjIzMzKuBCYmTU5FwIzsybnQmBm1uRcCMzMmpwLgZlZk3MhMDNrci4EZmZN7v8BhdBpPRweFBgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "while frame_idx < max_frames:\n",
    "    state = env.reset()\n",
    "    ou_noise.reset()\n",
    "    episode_reward = 0\n",
    "    \n",
    "    for step in range(max_steps):\n",
    "        action = policy_net.get_action(state)\n",
    "        action = ou_noise.get_action(action, step)\n",
    "        next_state, reward, done, _ = env.step(action)\n",
    "        \n",
    "        replay_buffer.push(state, action, reward, next_state, done)\n",
    "        if len(replay_buffer) > batch_size:\n",
    "            ddpg_update(batch_size)\n",
    "        \n",
    "        state = next_state\n",
    "        episode_reward += reward\n",
    "        frame_idx += 1\n",
    "        \n",
    "        if frame_idx % max(1000, max_steps + 1) == 0:\n",
    "            plot(frame_idx, rewards)\n",
    "        \n",
    "        if done:\n",
    "            break\n",
    "    \n",
    "    rewards.append(episode_reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch4]",
   "language": "python",
   "name": "conda-env-pytorch4-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
